################################
#	John Henderson
#	Gerrymandering Incumbency 
#		(with Brian Hamel and Aaron Goldzimer)
#		January 1, 2018
#
################################
# SD flip probability plots
################################

rm(list=ls())
library(stringr)

setwd('~/Dropbox/StateRedistricting/replication/short')

state=chamber=NULL
flips=NULL   
 
library(foreign) 
library(mice)  
library(matrixStats)

imputeNAplans=function(dists,plans=NULL){
	# imputing missing cells for partial plans
	# can't just align districts by number, since these have geographical 
	# definition; and don't have geographical boundaries for non-districts
	# in shapefiles, etc. 
	
	# hard and fast rule: sort districts on the pr(Obama 08) for whole plans, 
	# then match existing districts in partial plans to those that reduce the k-plan
	# distances
	if(is.null(plans)){
		warning('No index for plans is included: care is required to map back onto the various alternatives.')
	}   
	# length of datas
	m=length(dists)
	n=nrow(dists[[1]])  
	# length of completes
	cm=0
	#mu=array(NA,m)  
	is.cm=array(FALSE,m)  
	ord.dists=list() 
	ord.mat=matrix(NA,n,length(dists))
	for(i in 1:m){
		if(all(complete.cases(dists[[i]]))){
			cm=cm+1
			is.cm[i]=T
			mu=dists[[i]][,1]/(dists[[i]][,1]+dists[[i]][,2])
			ord.dists[[i]]=dists[[i]][order(mu),]
			
			ord.mat[,i]=t(t(sort(mu)))
		}
	}   
	
	for(i in 1:m){  
		if(is.cm[i]==F){
			m0=sort(dists[[i]][,1]/(dists[[i]][,1]+dists[[i]][,2]),na.last=T)
			n0=as.numeric(names(m0))
			n1=array(NA,n)
			for(p in 1:length(m0)){ 
				if(is.na(m0[p])){
					break()
				}
				dd=sqrt(rowSums((ord.mat-m0[p])^2,na.rm=T)) 
				if(length(which(!is.na(n1)))>0){
					dd[1:max(which(!is.na(n1)))]=NA
				}
				n1[which(dd==min(dd,na.rm=T))]=n0[p]
				
				#if dd object is approaching length(m0) *too fast*
				if(p<length(m0[!is.na(m0)]) & max(which(is.na(n1)))<length(m0)){
					n1=c(n1[-c(max(which(is.na(n1))))],NA)
				} 			      			
			}
			n1[which(is.na(n1))]=sample(n0[which(is.na(m0))],replace=F)	
			ord.dists[[i]]=dists[[i]][n1,]		   		
		}
	}
	 
	# quasi-randomization simulation : plans that have missing districts
	#  will have missings imputed for *all* the other plans that have those 
	#  districts : average district bias will be recorded ... 
	# complete plans are analyzed as is
	dist_insa=matrix(NA,n,1)
	dist_ins=matrix(NA,n,m) 
	colnames(dist_ins)=plans
	for(i in 1:m){  
		if(is.cm[i]==F){
			qn=ord.dists[[i]][,1]/(ord.dists[[i]][,1]+ord.dists[[i]][,2])-ord.dists[[i]][,2]/(ord.dists[[i]][,1]+ord.dists[[i]][,2])			
		  	mat_temp=matrix(qn,n,sum(is.cm))
			jj=0
			for(j in which(is.cm)){
				jj=jj+1
				qs=ord.dists[[j]][,1]/(ord.dists[[j]][,1]+ord.dists[[j]][,2])-ord.dists[[j]][,2]/(ord.dists[[j]][,1]+ord.dists[[j]][,2])							
				mat_temp[which(is.na(qn)),jj]=qs[which(is.na(qn))]				
			}                                                     
			colnames(mat_temp)=paste(plans[i],'sim',1:jj,sep='_')
			if(mean(complete.cases(dist_insa))==0){
				dist_insa=mat_temp
			} else {
				dist_insa=cbind(dist_insa,mat_temp)
			}
		} else {
			dist_ins[,i]=ord.dists[[i]][,1]/(ord.dists[[i]][,1]+ord.dists[[i]][,2])-ord.dists[[i]][,2]/(ord.dists[[i]][,1]+ord.dists[[i]][,2])			
		}
	} 
	if(mean(is.cm)<1){
		dist_ins=cbind(dist_ins[,which(is.cm==T)],dist_insa)
	}
	return(dist_ins) 
}
 
############################
#
# counterfactual maps 2010
#
############################
                 
source('analysisSTFun.R')  
         
states=c(
  "AL","AK","AZ","AR","CA",  "CO","CT","DE","FL",  "GA","HI","ID","IL","IN",  "IA","KS","KY","LA","ME",  "MT","NE","NV","NH","NJ", "OR",  
  "NM","NY","NC","ND","OH",  "OK","MD","MA","MI",  "MN","MS","MO","PA","RI",  "SC","SD","TN","TX","UT",  "VT","VA","WA","WV","WI", "WY")

states=sort(states)
     
plans=read.csv('redist_authority.csv',stringsAsFactors=F)    
plans10=plans[which(plans[,1]==2010),]
included=array(FALSE,nrow(plans10))
for(i in 1:length(states)){
	included[which(states[i]==plans10[,2])]=TRUE
}

plans10=plans10[included,]
plans10=plans10[order(plans10[,2]),]

plansCD10 = array(NA,length(states))
plansCD10[which(plans10$congress_court==1)]='C'
plansCD10[which(plans10$congress_independent==1)]='I'
plansCD10[which(plans10$congress_politician==1)]='B'
plansCD10[which(is.na(plansCD10) & plans10$congress_legislatue_partycontrol=='nonpartisan')]='B' # NE
plansCD10[which(is.na(plansCD10) & plans10$congress_legislatue_partycontrol=='split')]='B'
plansCD10[which(is.na(plansCD10) & plans10$congress_legislatue_partycontrol=='D')]='D'
plansCD10[which(is.na(plansCD10) & plans10$congress_legislatue_partycontrol=='R')]='R'
plansCD10[which(is.na(plansCD10))]='N'

plansAD10 = array(NA,length(states))
plansAD10[which(plans10$state_court==1)]='C'
plansAD10[which(plans10$state_independent==1)]='I'
plansAD10[which(plans10$state_politician==1)]='B'
plansAD10[which(is.na(plansAD10) & plans10$state_legislature_partycontrol=='nonpartisan')]='B' # NE
plansAD10[which(is.na(plansAD10) & plans10$state_legislature_partycontrol=='split')]='B'
plansAD10[which(is.na(plansAD10) & plans10$state_legislature_partycontrol=='D')]='D'
plansAD10[which(is.na(plansAD10) & plans10$state_legislature_partycontrol=='R')]='R'
plansAD10[which(is.na(plansAD10))]='N'
 
plansSD10=plansAD10
       
#############################
#
# sd
#
#############################
   
#############################  
## Florida has multiple plans and so need to adjust for the court map in 2015/2016
#############################
#folders=paste('SD_plans/',system('ls ~/Dropbox/StateRedistricting/Data/plans2010/FL/SD/SD_plans/',intern=T),sep='')
#full_plans=read.csv('~/Dropbox/StateRedistricting/Data/plans2010/FL/scraped/PlanDataIns.csv',header=F,stringsAsFactors=F)
#dates=array(NA,length(folders))
#folders=gsub(folders,pattern='SD_plans/',replace='') 
#for(i in 3:119){
#	ix=which(gsub(folders[i],pattern='_adopted',replace='')==full_plans[,1])
#	dates[i]=str_sub(full_plans[ix,3],7)
#}
#dates[1]='2015'
#dates[2]='2002'
#dates[120:length(dates)]='2015'   
#save(dates,folders,file='~/Dropbox/StateRedistricting/replication/short/SD_FL_dates.Rdata')
load('plans2010/SD_FL_dates.Rdata') 


######################
# flips on since recovers the partisan margins 
###################### 

load('stateSwings.Rdata') 

flipPr=function(temp,state,year_mins,year_maxs,seats=T){
 	# assumes that temp is built from rshare-(1-rshare)
	rshare=(temp+1)/2    
	dshare=1-(temp+1)/2  

	#imagine some feasible counterfactual
	i.state=which(rownames(year_mins)==state)    
	minR=year_mins[i.state,1]/100
	maxR=year_maxs[i.state,1]/100
	minD=year_mins[i.state,2]/100
	maxD=year_maxs[i.state,2]/100  

	updat=as.numeric((as.numeric(sign((dshare+abs(maxD-mean(dshare)))-(rshare-abs(minR-mean(rshare))))!=sign(dshare-rshare))+
	as.numeric(sign((rshare+abs(maxR-mean(rshare)))-(dshare-abs(minD-mean(dshare))))!=sign(rshare-dshare)))>0)

	if(seats==T){
		return(mean(updat))
	} else{
		dshare+abs(maxD-mean(dshare))
		rshare-abs(minR-mean(rshare))		

		rshare+abs(maxR-mean(rshare)) 	
		dshare-abs(minD-mean(dshare))
	}

}  

######################
# cd - predicted probability at CD 
######################  

#model seat control at CD by pres vote share w/n state and 2012/2014 year
load('voteData.Rdata')
# voteData
voteData$Rseat=as.numeric(voteData[,1]<voteData[,2])
voteData12=voteData[which(voteData$"YearElec"==2012),] 
voteData12$Sflip=as.numeric(voteData$Rseat[which(voteData$"YearElec"==2012)]!=voteData$Rseat[which(voteData$"YearElec"==2014)])

rm(voteData) 
voteData12=voteData12[order(voteData12$StPOAbrv,voteData12$RPresShr,decreasing=F),]

y=voteData12$Rseat
z=voteData12$Sflip

voteData12$pShare=array(NA,length(voteData12$"StPOAbrv"))     
un_st=unique(voteData12$"StPOAbrv") 
k_get=NULL  
flips=T  

un_st=c('AK','AZ','CA','CO','FL','ID','MT','NC','NM','NV','OH','SC','TX','VA','WA')  
for(i in 1:length(un_st)){
	state=un_st[i]  
	
	chamber='CD'

	#source('~/Dropbox/StateRedistricting/Code/plans2010/analysisST.R')
	if(state!='AK' & state!='MT' & state!='OH'){
		st=analysisST(state=state,chamber=chamber,flips=flips,m0=m0,m1=1000000)	
		k_get=st$k_get
		adopted=st$adopted
	}
	
	if(!is.null(k_get)){
		temp=(k_get[adopted[1],]+1)/2
		ik=which(voteData12$"StPOAbrv"==state)
		voteData12$pShare[ik]=temp
		##(sort(k_get[adopted[1],])+1)/2
	}
	k_get=NULL
}

voteData12$difpShare=abs((voteData12$pShare)-(1-voteData12$pShar))

#linear=lm(voteData12$Rseat~voteData12$pShare+as.factor(voteData12$StPOAbrv))    
#logit=glm(voteData12$Rseat~voteData12$pShare+as.factor(voteData12$StPOAbrv),family=binomial(link=logit))

logit=linear=list()    

linear[[1]]=lm(voteData12$Rseat~voteData12$pShare)    
logit[[1]]=glm(voteData12$Rseat~voteData12$pShare,family=binomial(link=logit))
                                                                               
linear[[2]]=lm(voteData12$Sflip~voteData12$difpShare)    
logit[[2]]=glm(voteData12$Sflip~voteData12$difpShare,family=binomial(link=logit))
        

######################
# cd - plots predicted probability => ad
######################
# turning on flips in analysisST.R
flips=T 
#xMat=voteData12[which(voteData12$St==state),] 

repMaj=function(temp,pred=T,linear,logit,model='logit',partisan=T){
	if(pred==F){
		tmp=mean(temp>0)
	} else if(pred==T){
		if(model=='logit'){
			if(partisan==T){
				tmp=(temp+1)/2		  	
			} else{
				tmp=abs(temp)
			}
			tmp=tmp*logit$coef[2]+logit$coef[1] 			
			tmp=1/(1+exp(-tmp))  
		} else if(model=='linear'){ 
			if(partisan==T){
				tmp=(temp+1)/2		  	
			} else{
				tmp=abs(temp)
			} 	  	
			tmp=tmp*linear$coef[2]+linear$coef[1] 			
			#tmp=1/(1+exp(-tmp))			
		}   		
	}    
	return(tmp)	
} 
      
states_cf=c('AK','AZ','CA','CO','FL','FL','ID','MT','NC','NM','NV','OH','SC','TX','VA','WA')    
ix=c(which(states=='AK'),
	which(states=='AZ'),
	which(states=='CA'),
	which(states=='CO'),	
	which(states=='FL'),
	which(states=='FL'),	
	which(states=='ID'), 
	which(states=='MT'), 
	which(states=='NC'),	 
	which(states=='NM'),
	which(states=='NV'),	 	
	which(states=='OH'),	
	which(states=='SC'),
	which(states=='TX'),
	which(states=='VA'),
	which(states=='WA'))
#

plansSD10=plansSD10[ix] 
#plansSD10[which(states_cf=='AK')]='I'
plansSD10[which(states_cf=='FL')][2]='C' 
plansSD10[which(states_cf=='OH')]='R' 
         
# state-level OH politician committee looks partisan to me... 
states_ix=states_cf[c(which(plansSD10=='D'),which(plansSD10=='R'),which(plansSD10=='B'),which(plansSD10=='C'),which(plansSD10=='I'))]     
# note: no state-level simulations 
 

# flip prob II
{
pdf(width=5,height=6.5,file='sd_2010-flipprob.pdf')
plot(x=-1000, y=-1000, type='n', xlim=c(-.075, .2), ylim=c(.5,length(states_ix)+.5), axes=F, ylab="", xlab="")#, main=paste("Margins of Victory in House Elections\n",state,sep=""))
axis(1,at=c(0,.1,.2),labels=c(0,.1,.2))
#cnts=0
#cnts=0
cnts=length(states_ix)+1       
nls=fls=0   
#Means and Medians
for (state in states_ix){
	cnts=cnts-1
    chamber='SD'
	#source('~/Dropbox/StateRedistricting/Code/plans2010/analysisST.R')
    # adopted & k_get are the relevant objects  
	# built in .. 
     
	st=analysisST(state=state,chamber=chamber,flips=flips,m0=m0,m1=1000000)	
	k_get=st$k_get
	adopted=st$adopted 

		
	if(state=='VA'){
		adopted=adopted[1]
	}         

	sts=state
	if(state=='FL' & fls==0){
		folds=folders[which(as.numeric(dates)<2015)] 
		inc=c()
		for(i in 1:length(folds)){
			inc=c(inc,grep(rownames(k_get),pattern=folds[i]))  
		}
		k_get=k_get[sort(inc),]		
		fls=fls+1
		adopted=grep(rownames(k_get),pattern='adopted') 		
		#sts=paste(sts,'\n(2011)',sep='')            
		yar=2011
	} else if(state=='FL' & fls==1){
		folds=folders[which(as.numeric(dates)>2014)]
		inc=c()
		for(i in 1:length(folds)){
			inc=c(inc,grep(rownames(k_get),pattern=folds[i]))  
		}
		k_get=k_get[sort(inc),]		
		adopted=grep(rownames(k_get),pattern='adopted')
		#sts=paste(sts,'\n(2015)',sep='')	 
		yar=2015
	}      
   
   
   	# remove those with extra but simular adopted plans
   	if(length(grep(rownames(k_get),pattern='adopted'))>1){
   		rownames(k_get)[grep(rownames(k_get),pattern='adopted')[-c(1)]]=gsub(rownames(k_get)[grep(rownames(k_get),pattern='adopted')[-c(1)]],pattern='adopted_',replace='')
   		adopted=adopted[1]
   	}  

   	temp.sims = k_get[-c(adopted),]
   	temp.10 = k_get[adopted,] 
 
	# smooth with a model, but need R seat control 
	means.sim=colMeans(apply(temp.sims,1,repMaj,model='logit',logit=logit[[2]],partisan=F))
		#means.sim=apply(temp.sims,1,repMaj,model='logit',logit=logit) 
	mean.10=mean(repMaj(temp=temp.10,model='logit',logit=logit[[2]],partisan=F))
		#mean.10=mean(repMaj(temp=temp.10,model='logit',logit=logit)>.5)
	ss.dif=mean(mean.10-means.sim)+.001

	if(length(ss.dif)>0){     
		#if(ss.dif>0){
		p.dif=1-sum(mean.10<means.sim)/length(means.sim)    
	} else{                                                     
		p.dif=1
		#sum(mean.10>means.sim)/length(means.sim)    
	} 

	cols='red' 
	ltys=2
	if(p.dif<.05){
		cols='red' 
		ltys=1
	}  

	if(length(mean.10)>0){ 	  
		points(x=means.sim, y=jitter(factor=.2,rep(cnts,length(means.sim))), col='darkblue', lwd=2,pch=20,cex=.52)
		points(y=rep(cnts,length(mean.10)), x=mean.10, col=cols, pch=1,cex=abs(ss.dif)*8+.58,lty=2) 			
		text(col='black',adj=c(0,NA),x=.005,y=cnts,labels=paste('(',str_sub(ss.dif,1,str_locate(ss.dif,pattern='[/.]')[1]+2),')',sep=''),srt = 1, pos = 2,cex=.5)				
    } else{
		text(col='black',adj=c(0,NA),x=.005,y=cnts,labels='NA',srt = 1, pos = 2,cex=.5)				
	} 
	text(adj=c(0,NA),x=-.0235,y=cnts,labels=as.character(state),srt = 1, pos = 2,cex=.65)	

	#text(adj=c(0,NA),x=-.055,y=cnts,labels=as.character(sts),srt = 1, pos = 2,cex=.65)   
	if(state=='FL'){
		text(adj=c(.5,.5),x=-.0235,y=-.28+cnts,labels=paste('(',as.character(yar),')',sep=''),srt = 1, pos = 2,cex=.475)   		
	}  
	#rm(k_get)
}         

# lines and legend graphics  
# lines and legend graphics   
lines(col='grey',lty=2,y=c(.5,.5),x=c(-.0215,1.))
lines(col='grey',lty=2,y=c(6.5,6.5),x=c(-.0215,1.))  
lines(col='grey',lty=2,y=c(10.5,10.5),x=c(-.0215,1.))   
lines(col='grey',lty=2,y=c(12.5,12.5),x=c(-.0215,1.))   
lines(col='grey',lty=2,y=c(16.5,16.5),x=c(-.0215,1.))

axis(2,line=-3.75,at=c(.5,6.5,10.5,12.5,16.5),labels=F,tck=-.02)

text(x=.2-.0105,y=16.05,labels='Republican',cex=.75,adj=c(1,1),srt=0)
text(x=.2-.0105,y=12.05,labels='Bipartisan',cex=.75,adj=c(1,1),srt=0) 
text(x=.2-.0105,y=10.05,labels='Court',cex=.75,adj=c(1,1),srt=0)     
text(x=.2-.0105,y=6.05,labels='Independent',cex=.75,adj=c(1,1),srt=0) 

mtext("Average Seat Flip Probability",side=1,at=.1,adj=.5,line=3)  



dev.off()
} 
# end counterfactual for SDS